import pandas as pd
import numpy as np
from sklearn.externals import joblib
import csv
import random
from random import choice

def random_list(start,stop,length):
    if length >= 0:
        length=int(length)
    start, stop = (int(start), int(stop)) if start <= stop else (int(stop), int(start))
    random_list = []
    for i in range(length):
        random_list.append(random.randint(start, stop))
    return random_list

# 从库中提取发布量大于MIN_ATI篇的作者不多于10人，作为训练样本
MIN_ATI = 2000
MAX_ATI = 2000

def order_dict(dicts, n):
    result = []
    result1 = []
    p = sorted([(k, v) for k, v in dicts.items()], reverse=True)
    s = set()
    for i in p:
        s.add(i[1])
    for i in sorted(s, reverse=True)[:n]:
        # if i < MIN_ATI or i > MAX_ATI:
        if i < MIN_ATI:
            continue
        for j in p:
            if j[1] == i:
                result.append(j)
    for r in result:
        result1.append(r[0])
    maxname = result1[0]
    minname = result1[-1]
    print(maxname)
    print(minname)
    return result1

csv_data = pd.read_csv('alldoc/corpora.csv')  # 读取训练数据
target_data = csv_data[['label']]
target_data_np = np.array(target_data)
namedict = {}
tag_times= {}
for i in range(0,target_data_np.size):
    item = target_data_np[i][0]
    if item not in tag_times:
        tag_times[item] = 1
    else:
        tag_times[item] += 1
# print(target_data_np[0])
# print(tag_times)
# print(namedict)

res= order_dict(tag_times, 200)  #对字典d={'a':1,'b':2}排序,返回['a']
# print(type(res))
# print(len(res))

foo = [0,1,2,3,4,5,6,7,8,9,10]
labellst = []
labeldict = {} #限制每个类别的训练数据的最大数量
test_labeldict ={}
MAX_COUNT = 800
TEST_MAX_COUNT = 100


with open("alldoc/corpora_train_4.csv", "w",newline='',encoding='utf-8') as csvfile_1:
    with open("alldoc/corpora_test_4.csv", "w", newline='', encoding='utf-8') as csvfile_2:  # 加newline= '',中间不会出现空行
        writer = csv.writer(csvfile_1)
        writer_test = csv.writer(csvfile_2)

    # 先写入columns_name
    # 特征1：文章长度
    # 特征2：句子长度
    # 特征3：平均词长
    # 特征4：文章的词汇丰富率
    # 特征5：包含不同成语的个数
    # 特征6：虚词的个数
    # 特征7：问号的个数
    # 特征8：感叹号的个数
    # 特征9：名词的个数
    # 特征10：动词的个数
    # writer.writerow(["txt_length", "ave_sentence_length", "ave_word_length", "richness_rate",
    #                  "idiom_count", "emptyword_count", "question_mark_count","exclamation_mark",
    #                   "noun_count", "verb_count","label"])

        writer.writerow(["txt_length", "ave_sentence_length", "ave_word_length", "richness_rate",
                         "idiom_count", "emptyword_count", "comma_count", "period_count", "question_mark_count",
                         "exclamation_mark", "colon_count", "semicolon_count", "punctuation_mark_count",
                         "noun_count", "verb_count", "label"])
        writer_test.writerow(["txt_length", "ave_sentence_length", "ave_word_length", "richness_rate",
                         "idiom_count", "emptyword_count", "comma_count", "period_count", "question_mark_count",
                         "exclamation_mark", "colon_count", "semicolon_count", "punctuation_mark_count",
                         "noun_count", "verb_count", "label"])

        with open('alldoc/corpora.csv', 'r', encoding='utf-8') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                if row['label'] in res:
                    random_num = choice(foo)
                    if random_num in [1,8]:  # 极小概率会命不中训练数据较少的样本
                    # if random_num < 6:
                        # if row['label'] in labeldict.keys():
                        #     labeldict[row['label']] += 1
                        #     if labeldict[row['label']] > MAX_COUNT:
                        #         continue
                        # else:
                        #     labeldict[row['label']] = 0
                        writer.writerow(row.values())
                        if row['label'] not in labellst:
                            labellst.append(row['label'])
                    elif random_num == 10:
                    # else:
                    #     if row['label'] in test_labeldict.keys():
                    #         test_labeldict[row['label']] += 1
                    #         if test_labeldict[row['label']] > TEST_MAX_COUNT:
                    #             continue
                    #     else:
                    #         test_labeldict[row['label']] = 0
                        writer_test.writerow(row.values())


fo = open('alldoc/corpora_train' + ".label_to_idx", "w")
for label in labellst:
    fo.write(label.replace(' ','') + "\n")
fo.close()


